{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "wbSGA8PNKvIy"
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "from matplotlib import rc\n",
    "plt.rc('font', family='serif')\n",
    "import tensorflow as tf\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RpuHMCndxqs4"
   },
   "outputs": [],
   "source": [
    "def scrape_folder(folder, num_iters, metric_names = ['psnr', 'ssim', 'lpips']):\n",
    "  stats = {}\n",
    "  for i_metric, metric_name in enumerate(metric_names):\n",
    "    filename = os.path.join(folder, 'test_preds', f'metric_{metric_name}_{num_iters}.txt')\n",
    "    with open(filename, mode='r') as f:\n",
    "      v = np.array([float(s) for s in f.readline().split(' ')])\n",
    "    stats[metric_name] = np.mean(v)\n",
    "\n",
    "  tic = time.time()\n",
    "  grab_tags = ['train_steps_per_sec', 'num_params', 'train_num_params']\n",
    "  grabbed_tags = {k:[] for k in grab_tags}\n",
    "  for pattern in ['events*']:#, 'eval/events*']:\n",
    "    for event_file in glob.glob(os.path.join(folder, pattern)):\n",
    "      for event in tf.compat.v1.train.summary_iterator(event_file):\n",
    "        value = event.summary.value\n",
    "        if len(value) > 0:\n",
    "          tag = event.summary.value[0].tag\n",
    "          if tag in grab_tags:\n",
    "            grabbed_tags[tag].append(np.array(tf.make_ndarray(event.summary.value[0].tensor)).item())\n",
    "\n",
    "  if grabbed_tags['train_steps_per_sec']:\n",
    "    steps_per_sec = np.percentile(np.array(grabbed_tags['train_steps_per_sec']), 95)\n",
    "    stats['num_hours'] = (num_iters / steps_per_sec) / (60**2)\n",
    "  else:\n",
    "    stats['num_hours'] = np.nan\n",
    "\n",
    "  if grabbed_tags['num_params']:\n",
    "    stats['mega_params'] = int(np.max(grabbed_tags['num_params'])) / (1000000) # in millions\n",
    "  elif grabbed_tags['train_num_params']:\n",
    "    stats['mega_params'] = int(np.max(grabbed_tags['train_num_params'])) / (1000000) # in millions\n",
    "  else:\n",
    "    stats['mega_params'] = np.nan\n",
    "\n",
    "  return stats\n",
    "\n",
    "def render_table(names, data, precisions, rank_order, suffixes=None, hlines = []):\n",
    "  def rankify(x, order):\n",
    "    assert len(x.shape) == 1\n",
    "    if order == 0:\n",
    "      return np.full_like(x, 1e5, dtype=np.int32)\n",
    "    u = np.sort(np.unique(x))\n",
    "    if order == 1:\n",
    "      u = u[::-1]\n",
    "    r = np.zeros_like(x, dtype=np.int32)\n",
    "    for ui, uu in enumerate(u):\n",
    "      mask = x == uu\n",
    "      r[mask] = ui\n",
    "    return np.int32(r)\n",
    "\n",
    "  tags = ['   \\cellcolor{red}',\n",
    "          '\\cellcolor{orange}',\n",
    "          '\\cellcolor{yellow}',\n",
    "          '                  ']\n",
    "\n",
    "  max_len = max([len(v) for v in list(names)])\n",
    "  names_padded = [v + ' '*(max_len-len(v)) for v in names]\n",
    "\n",
    "  data_quant = np.round((data * 10.**(np.array(precisions)[None, :]))) / 10.**(np.array(precisions)[None, :])\n",
    "  if suffixes is None:\n",
    "    suffixes = [''] * len(precisions)\n",
    "\n",
    "  tagranks = []\n",
    "  for d in range(data_quant.shape[1]):\n",
    "    tagranks.append(np.clip(rankify(data_quant[:,d], rank_order[d]), 0, len(tags)-1))\n",
    "  tagranks = np.stack(tagranks, -1)\n",
    "\n",
    "  for i_row in range(len(names)):\n",
    "    line = ''\n",
    "    if i_row in hlines:\n",
    "      line += '\\\\hline\\n'\n",
    "    line += names_padded[i_row]\n",
    "    for d in range(data_quant.shape[1]):\n",
    "      line += ' & '\n",
    "      if rank_order[d] != 0 and not np.isnan(data[i_row,d]):\n",
    "        line += tags[tagranks[i_row, d]]\n",
    "      if np.isnan(data[i_row,d]):\n",
    "        line += ' - '\n",
    "      else:\n",
    "        assert precisions[d] >= 0\n",
    "        line += ('{:' + f'0.{precisions[d]}f' + '}').format(data_quant[i_row,d]) + suffixes[d]\n",
    "    line += ' \\\\\\\\'\n",
    "    print(line)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b2wPHPku5jO8"
   },
   "source": [
    "Mip-NeRF 360's results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27352,
     "status": "ok",
     "timestamp": 1656870904226,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "gKnbxt2AKz8w",
    "outputId": "c8818082-cf6d-4212-c503-558e1dec49e7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mipnerf360_450936927/360 bicycle {'psnr': 24.39237983703613, 'ssim': 0.6856040573120117, 'lpips': 0.3024290585517883, 'num_hours': 6.2325243293076875, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 flowerbed {'psnr': 21.722528544339266, 'ssim': 0.5822233611887152, 'lpips': 0.3457577336918224, 'num_hours': 7.029125016907558, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 gardenvase {'psnr': 26.994542519251507, 'ssim': 0.8132077877720197, 'lpips': 0.16961431813736758, 'num_hours': 6.373895524145641, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 stump {'psnr': 26.433189034461975, 'ssim': 0.7453256547451019, 'lpips': 0.2610513372346759, 'num_hours': 6.372994107616741, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 treehill {'psnr': 22.80838351779514, 'ssim': 0.6286024749279022, 'lpips': 0.3419443451695972, 'num_hours': 6.373792756813197, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 fulllivingroom {'psnr': 31.74024293361566, 'ssim': 0.9143212055548643, 'lpips': 0.210464691504454, 'num_hours': 6.229779091791955, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 kitchencounter {'psnr': 29.52976067860921, 'ssim': 0.8942874411741892, 'lpips': 0.20467615375916162, 'num_hours': 6.376739565840228, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 kitchenlego {'psnr': 32.162456675938195, 'ssim': 0.9199009571756636, 'lpips': 0.12717486768960953, 'num_hours': 6.381604252968429, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 officebonsai {'psnr': 33.498427004427526, 'ssim': 0.9409479949925397, 'lpips': 0.1763693257360845, 'num_hours': 6.371603406297137, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360 {'psnr': 27.697990082830515, 'ssim': 0.7916023260936674, 'lpips': 0.23772020349717346, 'num_hours': 6.415784227965397, 'mega_params': 9.007493}\n",
      "mipnerf360_450936927/360_glo4 bicycle {'psnr': 24.053745727539063, 'ssim': 0.6909286546707153, 'lpips': 0.2945095646381378, 'num_hours': 6.3832255160561, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 flowerbed {'psnr': 21.374122966419566, 'ssim': 0.5803043625571511, 'lpips': 0.3430247970602729, 'num_hours': 6.38164317846884, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 gardenvase {'psnr': 25.532458384831745, 'ssim': 0.8063657606641451, 'lpips': 0.17009872818986574, 'num_hours': 6.386194917543813, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 stump {'psnr': 26.132395386695862, 'ssim': 0.7474702708423138, 'lpips': 0.25648255459964275, 'num_hours': 6.386754034296546, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 treehill {'psnr': 22.360591994391548, 'ssim': 0.6240272356404198, 'lpips': 0.33734683361318374, 'num_hours': 6.382674989177764, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 fulllivingroom {'psnr': 29.69650620680589, 'ssim': 0.912017149802966, 'lpips': 0.20836940980874574, 'num_hours': 6.387421971615129, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 kitchencounter {'psnr': 28.329934310913085, 'ssim': 0.8888227045536041, 'lpips': 0.20833696722984313, 'num_hours': 6.386267560219862, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 kitchenlego {'psnr': 30.207322692871095, 'ssim': 0.9143631560461861, 'lpips': 0.13056355544498988, 'num_hours': 6.387698179101134, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 officebonsai {'psnr': 30.50528634561075, 'ssim': 0.9331820059467006, 'lpips': 0.18204631998732285, 'num_hours': 6.254863324211862, 'mega_params': 9.012005}\n",
      "mipnerf360_450936927/360_glo4 {'psnr': 26.465818224008732, 'ssim': 0.7886090334138002, 'lpips': 0.2367531922857783, 'num_hours': 6.37074929674345, 'mega_params': 9.012005}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/nerf/'\n",
    "scene_names = ['bicycle', 'flowerbed', 'gardenvase', 'stump', 'treehill', 'fulllivingroom', 'kitchencounter', 'kitchenlego', 'officebonsai']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "models_meta[None] = 'mip-NeRF 360 (from the paper)'\n",
    "models_meta['mipnerf360_450936927/360'] = 'mip-NeRF 360'\n",
    "models_meta['mipnerf360_450936927/360_glo4'] = 'mip-NeRF 360 w/GLO'\n",
    "\n",
    "\n",
    "NUM_ITERS = 250000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for model_path in models_meta.keys():\n",
    "\n",
    "  if model_path is None:\n",
    "    # Inject the numbers from the paper.\n",
    "    psnrs = [24.37, 21.73, 26.98, 26.40, 22.87, 31.63, 29.55, 32.23, 33.46]\n",
    "    ssims = [0.685, 0.583, 0.813, 0.744, 0.632, 0.913, 0.894, 0.920, 0.941]\n",
    "    lpips = [0.301, 0.344, 0.170, 0.261, 0.339, 0.211, 0.204, 0.127, 0.176]\n",
    "    train_times = [np.nan]*len(psnrs)\n",
    "    model_sizes = [np.nan]*len(psnrs)\n",
    "    scene_stats = []\n",
    "    for p, s, l, tt, ms in zip(psnrs, ssims, lpips, train_times, model_sizes):\n",
    "      scene_stats.append({'psnr': p, 'ssim': s, 'lpips': l, 'num_hours': tt, 'mega_params': ms})\n",
    "    avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "    all_stats.append(scene_stats)\n",
    "    continue;\n",
    "\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS)\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 68,
     "status": "ok",
     "timestamp": 1656870904417,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "ISRu0l-f0kTU",
    "outputId": "0674ba64-e15e-450c-d8d1-77f6f740ca48"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mip-NeRF 360 (from the paper) & \\cellcolor{orange}27.69 &    \\cellcolor{red}0.792 &    \\cellcolor{red}0.237 &  -  &  -  \\\\\n",
      "mip-NeRF 360                  &    \\cellcolor{red}27.70 &    \\cellcolor{red}0.792 & \\cellcolor{orange}0.238 & 6.42 & 9.0M \\\\\n",
      "mip-NeRF 360 w/GLO            & \\cellcolor{yellow}26.47 & \\cellcolor{orange}0.789 &    \\cellcolor{red}0.237 & 6.37 & 9.0M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 1]\n",
    "rank_order = [1, 1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 61,
     "status": "ok",
     "timestamp": 1656870904594,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "6FoznCOYTQUf",
    "outputId": "8154ca71-8949-4de9-c567-5ef73e98a812"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "psnr\n",
      " & \\textit{bicycle} & \\textit{flowers} & \\textit{garden} & \\textit{stump} & \\textit{treehill} & \\textit{room} & \\textit{counter} & \\textit{kitchen} & \\textit{bonsai} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (from the paper) & \\cellcolor{orange}24.37 &    \\cellcolor{red}21.73 & \\cellcolor{orange}26.98 & \\cellcolor{orange}26.40 &    \\cellcolor{red}22.87 & \\cellcolor{orange}31.63 &    \\cellcolor{red}29.55 &    \\cellcolor{red}32.23 & \\cellcolor{orange}33.46 \\\\\n",
      "mip-NeRF 360                  &    \\cellcolor{red}24.39 & \\cellcolor{orange}21.72 &    \\cellcolor{red}26.99 &    \\cellcolor{red}26.43 & \\cellcolor{orange}22.81 &    \\cellcolor{red}31.74 & \\cellcolor{orange}29.53 & \\cellcolor{orange}32.16 &    \\cellcolor{red}33.50 \\\\\n",
      "mip-NeRF 360 w/GLO            & \\cellcolor{yellow}24.05 & \\cellcolor{yellow}21.37 & \\cellcolor{yellow}25.53 & \\cellcolor{yellow}26.13 & \\cellcolor{yellow}22.36 & \\cellcolor{yellow}29.70 & \\cellcolor{yellow}28.33 & \\cellcolor{yellow}30.21 & \\cellcolor{yellow}30.51 \\\\\n",
      "\n",
      "ssim\n",
      " & \\textit{bicycle} & \\textit{flowers} & \\textit{garden} & \\textit{stump} & \\textit{treehill} & \\textit{room} & \\textit{counter} & \\textit{kitchen} & \\textit{bonsai} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (from the paper) & \\cellcolor{yellow}0.685 &    \\cellcolor{red}0.583 &    \\cellcolor{red}0.813 & \\cellcolor{yellow}0.744 &    \\cellcolor{red}0.632 & \\cellcolor{orange}0.913 &    \\cellcolor{red}0.894 &    \\cellcolor{red}0.920 &    \\cellcolor{red}0.941 \\\\\n",
      "mip-NeRF 360                  & \\cellcolor{orange}0.686 & \\cellcolor{orange}0.582 &    \\cellcolor{red}0.813 & \\cellcolor{orange}0.745 & \\cellcolor{orange}0.629 &    \\cellcolor{red}0.914 &    \\cellcolor{red}0.894 &    \\cellcolor{red}0.920 &    \\cellcolor{red}0.941 \\\\\n",
      "mip-NeRF 360 w/GLO            &    \\cellcolor{red}0.691 & \\cellcolor{yellow}0.580 & \\cellcolor{orange}0.806 &    \\cellcolor{red}0.747 & \\cellcolor{yellow}0.624 & \\cellcolor{yellow}0.912 & \\cellcolor{orange}0.889 & \\cellcolor{orange}0.914 & \\cellcolor{orange}0.933 \\\\\n",
      "\n",
      "lpips\n",
      " & \\textit{bicycle} & \\textit{flowers} & \\textit{garden} & \\textit{stump} & \\textit{treehill} & \\textit{room} & \\textit{counter} & \\textit{kitchen} & \\textit{bonsai} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (from the paper) & \\cellcolor{orange}0.301 & \\cellcolor{orange}0.344 &    \\cellcolor{red}0.170 & \\cellcolor{orange}0.261 & \\cellcolor{orange}0.339 & \\cellcolor{yellow}0.211 &    \\cellcolor{red}0.204 &    \\cellcolor{red}0.127 &    \\cellcolor{red}0.176 \\\\\n",
      "mip-NeRF 360                  & \\cellcolor{yellow}0.302 & \\cellcolor{yellow}0.346 &    \\cellcolor{red}0.170 & \\cellcolor{orange}0.261 & \\cellcolor{yellow}0.342 & \\cellcolor{orange}0.210 & \\cellcolor{orange}0.205 &    \\cellcolor{red}0.127 &    \\cellcolor{red}0.176 \\\\\n",
      "mip-NeRF 360 w/GLO            &    \\cellcolor{red}0.295 &    \\cellcolor{red}0.343 &    \\cellcolor{red}0.170 &    \\cellcolor{red}0.256 &    \\cellcolor{red}0.337 &    \\cellcolor{red}0.208 & \\cellcolor{yellow}0.208 & \\cellcolor{orange}0.131 & \\cellcolor{orange}0.182 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3]\n",
    "rank_orders = [1, 1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "name_map['gardenvase'] = 'garden'\n",
    "name_map['flowerbed'] = 'flowers'\n",
    "name_map['fulllivingroom'] = 'room'\n",
    "name_map['kitchencounter'] = 'counter'\n",
    "name_map['kitchenlego'] = 'kitchen'\n",
    "name_map['officebonsai'] = 'bonsai'\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['psnr', 'ssim', 'lpips']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names), hlines = [len(names)-3])\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 13540,
     "status": "ok",
     "timestamp": 1656870918242,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "wpFu_zPZV8rM",
    "outputId": "bc6e8748-0017-4452-8262-956f4f31c1d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mipnerf360_452961962/blender_256 chair {'psnr': 35.14409606933594, 'ssim': 0.9807422143220902, 'lpips': 0.019885611250065268, 'num_hours': 3.0120813620093627, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 drums {'psnr': 25.672427253723143, 'ssim': 0.9345874139666557, 'lpips': 0.06211544882506132, 'num_hours': 2.454567881500612, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 ficus {'psnr': 32.68634537696838, 'ssim': 0.9779657471179962, 'lpips': 0.022341654361225663, 'num_hours': 2.4591483474261193, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 hotdog {'psnr': 37.58191083908081, 'ssim': 0.9816449856758118, 'lpips': 0.024997519506141545, 'num_hours': 2.4588705980222576, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 lego {'psnr': 36.30012715339661, 'ssim': 0.9804214736819268, 'lpips': 0.01723310980014503, 'num_hours': 3.00943955424293, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 materials {'psnr': 30.475751152038573, 'ssim': 0.9558810117840767, 'lpips': 0.04359573864378035, 'num_hours': 3.0132687091659123, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 mic {'psnr': 36.44918758392334, 'ssim': 0.9910019421577454, 'lpips': 0.008599376350175589, 'num_hours': 3.0127024649051144, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 ship {'psnr': 30.35776879310608, 'ssim': 0.8874139302968979, 'lpips': 0.12848058927804232, 'num_hours': 3.0069640380480425, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_256 {'psnr': 33.08345177769661, 'ssim': 0.9612073398754, 'lpips': 0.04090613100182963, 'num_hours': 2.803380369415044, 'mega_params': 0.835205}\n",
      "mipnerf360_452961962/blender_512 chair {'psnr': 35.857433795928955, 'ssim': 0.98388851583004, 'lpips': 0.01675196835771203, 'num_hours': 3.7615084467568956, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 drums {'psnr': 25.507867193222047, 'ssim': 0.9318369966745377, 'lpips': 0.06635790368542076, 'num_hours': 3.7576099485237027, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 ficus {'psnr': 33.011657524108884, 'ssim': 0.9789321529865265, 'lpips': 0.021975036268122493, 'num_hours': 3.760496131362062, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 hotdog {'psnr': 38.04464369773865, 'ssim': 0.9834308186173439, 'lpips': 0.021500862692482768, 'num_hours': 3.7610453555148697, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 lego {'psnr': 36.325964279174805, 'ssim': 0.9808579578995704, 'lpips': 0.01648236089386046, 'num_hours': 4.3412879804404625, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 materials {'psnr': 30.130296745300292, 'ssim': 0.9524855437874794, 'lpips': 0.04783135158009827, 'num_hours': 4.343275342164319, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 mic {'psnr': 36.847799701690676, 'ssim': 0.9914190286397934, 'lpips': 0.00919440199737437, 'num_hours': 4.337300623881409, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 ship {'psnr': 31.14282497406006, 'ssim': 0.8940365880727768, 'lpips': 0.11575569273903966, 'num_hours': 3.759209558641999, 'mega_params': 3.065733}\n",
      "mipnerf360_452961962/blender_512 {'psnr': 33.35856098890305, 'ssim': 0.9621109503135086, 'lpips': 0.03948119727676385, 'num_hours': 3.977716673410715, 'mega_params': 3.065733}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/nerf/'\n",
    "scene_names = ['chair', 'drums', 'ficus', 'hotdog', 'lego', 'materials', 'mic', 'ship']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "\n",
    "models_meta['mipnerf360_452961962/blender_256'] = 'mip-NeRF 360 (256 hidden)'\n",
    "models_meta['mipnerf360_452961962/blender_512'] = 'mip-NeRF 360 (512 hidden)'\n",
    "\n",
    "NUM_ITERS = 250000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for model_path in models_meta.keys():\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS)\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 70,
     "status": "ok",
     "timestamp": 1656870918436,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "knNz0h93yzXP",
    "outputId": "10364ad3-dcba-4549-a1d5-5b7380eef1d0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}33.08 & \\cellcolor{orange}0.961 & \\cellcolor{orange}0.041 & 2.80 & 0.8M \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}33.36 &    \\cellcolor{red}0.962 &    \\cellcolor{red}0.039 & 3.98 & 3.1M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 1]\n",
    "rank_order = [1, 1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 68,
     "status": "ok",
     "timestamp": 1656870918615,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "62it2f5ukbMP",
    "outputId": "1b0d078a-b2dc-4291-c373-6f58f2489742"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "psnr\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}35.14 &    \\cellcolor{red}25.67 & \\cellcolor{orange}32.69 & \\cellcolor{orange}37.58 & \\cellcolor{orange}36.30 &    \\cellcolor{red}30.48 & \\cellcolor{orange}36.45 & \\cellcolor{orange}30.36 \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}35.86 & \\cellcolor{orange}25.51 &    \\cellcolor{red}33.01 &    \\cellcolor{red}38.04 &    \\cellcolor{red}36.33 & \\cellcolor{orange}30.13 &    \\cellcolor{red}36.85 &    \\cellcolor{red}31.14 \\\\\n",
      "\n",
      "ssim\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}0.981 &    \\cellcolor{red}0.935 & \\cellcolor{orange}0.978 & \\cellcolor{orange}0.982 & \\cellcolor{orange}0.980 &    \\cellcolor{red}0.956 &    \\cellcolor{red}0.991 & \\cellcolor{orange}0.887 \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}0.984 & \\cellcolor{orange}0.932 &    \\cellcolor{red}0.979 &    \\cellcolor{red}0.983 &    \\cellcolor{red}0.981 & \\cellcolor{orange}0.952 &    \\cellcolor{red}0.991 &    \\cellcolor{red}0.894 \\\\\n",
      "\n",
      "lpips\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}0.020 &    \\cellcolor{red}0.062 &    \\cellcolor{red}0.022 & \\cellcolor{orange}0.025 & \\cellcolor{orange}0.017 &    \\cellcolor{red}0.044 &    \\cellcolor{red}0.009 & \\cellcolor{orange}0.128 \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}0.017 & \\cellcolor{orange}0.066 &    \\cellcolor{red}0.022 &    \\cellcolor{red}0.022 &    \\cellcolor{red}0.016 & \\cellcolor{orange}0.048 &    \\cellcolor{red}0.009 &    \\cellcolor{red}0.116 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3]\n",
    "rank_orders = [1, 1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['psnr', 'ssim', 'lpips']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names), hlines = [len(names)-2])\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 14157,
     "status": "ok",
     "timestamp": 1656870932910,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "8Dq0kS3jzLQu",
    "outputId": "657cfa79-8959-4f6b-bcb1-517a2f57e3ae"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mipnerf360_452868940/llff_256 fern {'psnr': 25.53523317972819, 'ssim': 0.8355980316797892, 'lpips': 0.17796580493450165, 'num_hours': 3.0791895901515773, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 flower {'psnr': 28.27823829650879, 'ssim': 0.8683305859565735, 'lpips': 0.13373453170061111, 'num_hours': 2.430556049280797, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 fortress {'psnr': 31.65355904897054, 'ssim': 0.9036512076854706, 'lpips': 0.10307400052746137, 'num_hours': 2.420831934334721, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 horns {'psnr': 28.028242111206055, 'ssim': 0.8812565580010414, 'lpips': 0.14718765299767256, 'num_hours': 3.0909099844438597, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 leaves {'psnr': 21.118935585021973, 'ssim': 0.7481956332921982, 'lpips': 0.18846691772341728, 'num_hours': 3.099055060584408, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 orchids {'psnr': 20.187190532684326, 'ssim': 0.682639941573143, 'lpips': 0.2243024781346321, 'num_hours': 2.4169084620651113, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 room {'psnr': 33.111086209615074, 'ssim': 0.9573092659314474, 'lpips': 0.08938046048084895, 'num_hours': 2.4188344266999375, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 trex {'psnr': 27.265713827950613, 'ssim': 0.9158718841416496, 'lpips': 0.11908453490052905, 'num_hours': 3.0837948697070785, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_256 {'psnr': 26.897274848960695, 'ssim': 0.8491066385326641, 'lpips': 0.14789954767495925, 'num_hours': 2.7550100471584367, 'mega_params': 0.835205}\n",
      "mipnerf360_452868940/llff_512 fern {'psnr': 25.259475072224934, 'ssim': 0.8444480299949646, 'lpips': 0.15365532040596008, 'num_hours': 3.7073142281576246, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 flower {'psnr': 28.34323196411133, 'ssim': 0.8737815976142883, 'lpips': 0.11761842519044877, 'num_hours': 4.416412998877655, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 fortress {'psnr': 31.545193036397297, 'ssim': 0.9064980645974478, 'lpips': 0.0853491226832072, 'num_hours': 4.410339683910223, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 horns {'psnr': 28.975930213928223, 'ssim': 0.9083132669329643, 'lpips': 0.10360320564359426, 'num_hours': 4.411220554963127, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 leaves {'psnr': 20.930516242980957, 'ssim': 0.757549062371254, 'lpips': 0.1729690581560135, 'num_hours': 4.405542887670279, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 orchids {'psnr': 19.98445224761963, 'ssim': 0.6835718005895615, 'lpips': 0.21904940903186798, 'num_hours': 4.416696944857583, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 room {'psnr': 33.66789722442627, 'ssim': 0.9654540717601776, 'lpips': 0.06235106599827608, 'num_hours': 4.409909662392265, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 trex {'psnr': 28.37834221976144, 'ssim': 0.929371850831168, 'lpips': 0.10232165455818176, 'num_hours': 4.407786158934037, 'mega_params': 3.065733}\n",
      "mipnerf360_452868940/llff_512 {'psnr': 27.13562977768126, 'ssim': 0.8586234680864783, 'lpips': 0.1271146577084437, 'num_hours': 4.323152889970349, 'mega_params': 3.065733}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/nerf/'\n",
    "scene_names = ['fern', 'flower', 'fortress', 'horns', 'leaves', 'orchids', 'room', 'trex']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "models_meta['mipnerf360_452868940/llff_256'] = 'mip-NeRF 360 (256 hidden)'\n",
    "models_meta['mipnerf360_452868940/llff_512'] = 'mip-NeRF 360 (512 hidden)'\n",
    "\n",
    "\n",
    "NUM_ITERS = 250000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for i_model, model_path in enumerate(models_meta.keys()):\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS)\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 66,
     "status": "ok",
     "timestamp": 1656870933094,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "wiF0Y8mRzSUf",
    "outputId": "d01cec60-a22e-41e9-d268-dd8d449fcc67"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}26.90 & \\cellcolor{orange}0.849 & \\cellcolor{orange}0.148 & 2.76 & 0.8M \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}27.14 &    \\cellcolor{red}0.859 &    \\cellcolor{red}0.127 & 4.32 & 3.1M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 1]\n",
    "rank_order = [1, 1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 63,
     "status": "ok",
     "timestamp": 1656870933280,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "B8iQz1uBj_BM",
    "outputId": "f5a04137-0a8d-495b-9855-04c4fde7d77e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "psnr\n",
      " & \\textit{fern} & \\textit{flower} & \\textit{fortress} & \\textit{horns} & \\textit{leaves} & \\textit{orchids} & \\textit{room} & \\textit{t-rex} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) &    \\cellcolor{red}25.54 & \\cellcolor{orange}28.28 &    \\cellcolor{red}31.65 & \\cellcolor{orange}28.03 &    \\cellcolor{red}21.12 &    \\cellcolor{red}20.19 & \\cellcolor{orange}33.11 & \\cellcolor{orange}27.27 \\\\\n",
      "mip-NeRF 360 (512 hidden) & \\cellcolor{orange}25.26 &    \\cellcolor{red}28.34 & \\cellcolor{orange}31.55 &    \\cellcolor{red}28.98 & \\cellcolor{orange}20.93 & \\cellcolor{orange}19.98 &    \\cellcolor{red}33.67 &    \\cellcolor{red}28.38 \\\\\n",
      "\n",
      "ssim\n",
      " & \\textit{fern} & \\textit{flower} & \\textit{fortress} & \\textit{horns} & \\textit{leaves} & \\textit{orchids} & \\textit{room} & \\textit{t-rex} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}0.836 & \\cellcolor{orange}0.868 & \\cellcolor{orange}0.904 & \\cellcolor{orange}0.881 & \\cellcolor{orange}0.748 & \\cellcolor{orange}0.683 & \\cellcolor{orange}0.957 & \\cellcolor{orange}0.916 \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}0.844 &    \\cellcolor{red}0.874 &    \\cellcolor{red}0.906 &    \\cellcolor{red}0.908 &    \\cellcolor{red}0.758 &    \\cellcolor{red}0.684 &    \\cellcolor{red}0.965 &    \\cellcolor{red}0.929 \\\\\n",
      "\n",
      "lpips\n",
      " & \\textit{fern} & \\textit{flower} & \\textit{fortress} & \\textit{horns} & \\textit{leaves} & \\textit{orchids} & \\textit{room} & \\textit{t-rex} \\\\\\hline\n",
      "\\hline\n",
      "mip-NeRF 360 (256 hidden) & \\cellcolor{orange}0.178 & \\cellcolor{orange}0.134 & \\cellcolor{orange}0.103 & \\cellcolor{orange}0.147 & \\cellcolor{orange}0.188 & \\cellcolor{orange}0.224 & \\cellcolor{orange}0.089 & \\cellcolor{orange}0.119 \\\\\n",
      "mip-NeRF 360 (512 hidden) &    \\cellcolor{red}0.154 &    \\cellcolor{red}0.118 &    \\cellcolor{red}0.085 &    \\cellcolor{red}0.104 &    \\cellcolor{red}0.173 &    \\cellcolor{red}0.219 &    \\cellcolor{red}0.062 &    \\cellcolor{red}0.102 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3]\n",
    "rank_orders = [1, 1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "name_map['trex'] = 't-rex'\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['psnr', 'ssim', 'lpips']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names), hlines = [len(names)-2])\n",
    "  print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JRakQXoa5c_s"
   },
   "source": [
    "Reproducing Ref-NeRF's Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 6652,
     "status": "ok",
     "timestamp": 1656870940046,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "8W1_8o5cpdnP",
    "outputId": "1d52a6be-be7d-415a-fac4-681b5594e1f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blender_refnerf_453115176 chair {'psnr': 35.43609601974487, 'ssim': 0.9821090731024742, 'lpips': 0.019048675729427488, 'normals_mae': 20.85330159, 'num_hours': 6.571275680175564, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 drums {'psnr': 25.852998418807985, 'ssim': 0.9371991902589798, 'lpips': 0.06024521112442017, 'normals_mae': 27.641556060000003, 'num_hours': 6.567777006412732, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 ficus {'psnr': 30.604970512390135, 'ssim': 0.966888021826744, 'lpips': 0.03917563056573272, 'normals_mae': 40.80395823, 'num_hours': 6.5770709353398535, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 hotdog {'psnr': 37.485798597335815, 'ssim': 0.9824606701731682, 'lpips': 0.02336025163065642, 'normals_mae': 8.700470121, 'num_hours': 6.570256454244849, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 lego {'psnr': 36.05616488456726, 'ssim': 0.9799841883778572, 'lpips': 0.01861008478794247, 'normals_mae': 24.550344864999996, 'num_hours': 6.577586022517292, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 materials {'psnr': 35.035994396209716, 'ssim': 0.9813295957446099, 'lpips': 0.023410547198727726, 'normals_mae': 10.3090648925, 'num_hours': 6.568449424984051, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 mic {'psnr': 36.72967861175537, 'ssim': 0.9917984154820442, 'lpips': 0.007707923451671377, 'normals_mae': 23.699775159999998, 'num_hours': 6.5744581258500725, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 ship {'psnr': 30.507489061355592, 'ssim': 0.8849879172444344, 'lpips': 0.13231919281184673, 'normals_mae': 30.48735126, 'num_hours': 6.570066162619403, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 {'psnr': 33.46364881277084, 'ssim': 0.963344634026289, 'lpips': 0.04048468966255314, 'normals_mae': 23.3807277723125, 'num_hours': 6.5721174765179775, 'mega_params': 0.71323}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/dorverbin/nerf/mipnerf360/'\n",
    "scene_names = ['chair', 'drums', 'ficus', 'hotdog', 'lego', 'materials', 'mic', 'ship']\n",
    "metric_names = ['psnr', 'ssim', 'lpips', 'normals_mae']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "models_meta[None] = 'ref-NeRF (from the paper)'\n",
    "models_meta['blender_refnerf_453115176'] = 'ref-NeRF 360'\n",
    "\n",
    "NUM_ITERS = 250000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for model_path in models_meta.keys():\n",
    "\n",
    "  if model_path is None:\n",
    "    # Inject the numbers from the paper.\n",
    "    psnrs = [35.83, 25.79, 33.91, 37.72, 36.25, 35.41, 36.76, 30.28]\n",
    "    ssims = [0.984, 0.937, 0.983, 0.984, 0.981, 0.983, 0.992, 0.880]\n",
    "    lpips = [0.017, 0.059, 0.019, 0.022, 0.018, 0.022, 0.007, 0.139]\n",
    "    # There was a bug in MAE computation for ['ficus', 'lego', 'mic'] in the paper, those numbers shouldn't be trusted.\n",
    "    nmaes = [19.852, 27.853, np.nan, 13.211, np.nan, 9.531, np.nan, 31.707]\n",
    "    train_times = [np.nan]*len(psnrs)\n",
    "    model_sizes = [np.nan]*len(psnrs)\n",
    "    scene_stats = []\n",
    "    for p, s, l, n, tt, ms in zip(psnrs, ssims, lpips, nmaes, train_times, model_sizes):\n",
    "      scene_stats.append({'psnr': p, 'ssim': s, 'lpips': l, 'normals_mae': n, 'num_hours': tt, 'mega_params': ms})\n",
    "    avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "    all_stats.append(scene_stats)\n",
    "    continue;\n",
    "\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS, metric_names=metric_names)\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 67,
     "status": "ok",
     "timestamp": 1656870940232,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "EPQZUmp7p2N1",
    "outputId": "3ee94963-dd84-4025-b61f-21c58d8f2b84"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ref-NeRF (from the paper) &    \\cellcolor{red}33.99 &    \\cellcolor{red}0.966 &    \\cellcolor{red}0.038 &  -  &  -  &  -  \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}33.46 & \\cellcolor{orange}0.963 & \\cellcolor{orange}0.040 &    \\cellcolor{red}23.38 & 6.57 & 0.7M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 2, 1]\n",
    "rank_order = [1, 1, -1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 72,
     "status": "ok",
     "timestamp": 1656870940411,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "8dZGlK7iKI48",
    "outputId": "b2c70244-b4b2-4e93-9fa1-1706053b2a1c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "psnr\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "ref-NeRF (from the paper) &    \\cellcolor{red}35.83 & \\cellcolor{orange}25.79 &    \\cellcolor{red}33.91 &    \\cellcolor{red}37.72 &    \\cellcolor{red}36.25 &    \\cellcolor{red}35.41 &    \\cellcolor{red}36.76 & \\cellcolor{orange}30.28 \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}35.44 &    \\cellcolor{red}25.85 & \\cellcolor{orange}30.60 & \\cellcolor{orange}37.49 & \\cellcolor{orange}36.06 & \\cellcolor{orange}35.04 & \\cellcolor{orange}36.73 &    \\cellcolor{red}30.51 \\\\\n",
      "\n",
      "ssim\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "ref-NeRF (from the paper) &    \\cellcolor{red}0.984 &    \\cellcolor{red}0.937 &    \\cellcolor{red}0.983 &    \\cellcolor{red}0.984 &    \\cellcolor{red}0.981 &    \\cellcolor{red}0.983 &    \\cellcolor{red}0.992 & \\cellcolor{orange}0.880 \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}0.982 &    \\cellcolor{red}0.937 & \\cellcolor{orange}0.967 & \\cellcolor{orange}0.982 & \\cellcolor{orange}0.980 & \\cellcolor{orange}0.981 &    \\cellcolor{red}0.992 &    \\cellcolor{red}0.885 \\\\\n",
      "\n",
      "lpips\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "ref-NeRF (from the paper) &    \\cellcolor{red}0.017 &    \\cellcolor{red}0.059 &    \\cellcolor{red}0.019 &    \\cellcolor{red}0.022 &    \\cellcolor{red}0.018 &    \\cellcolor{red}0.022 &    \\cellcolor{red}0.007 & \\cellcolor{orange}0.139 \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}0.019 & \\cellcolor{orange}0.060 & \\cellcolor{orange}0.039 & \\cellcolor{orange}0.023 & \\cellcolor{orange}0.019 & \\cellcolor{orange}0.023 & \\cellcolor{orange}0.008 &    \\cellcolor{red}0.132 \\\\\n",
      "\n",
      "normals_mae\n",
      " & \\textit{chair} & \\textit{drums} & \\textit{ficus} & \\textit{hotdog} & \\textit{lego} & \\textit{materials} & \\textit{mic} & \\textit{ship} \\\\\\hline\n",
      "ref-NeRF (from the paper) &    \\cellcolor{red}19.85 & \\cellcolor{orange}27.85 &  -  & \\cellcolor{orange}13.21 &  -  &    \\cellcolor{red}9.53 &  -  & \\cellcolor{orange}31.71 \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}20.85 &    \\cellcolor{red}27.64 &    \\cellcolor{red}40.80 &    \\cellcolor{red}8.70 &    \\cellcolor{red}24.55 & \\cellcolor{orange}10.31 &    \\cellcolor{red}23.70 &    \\cellcolor{red}30.49 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3, 2]\n",
    "rank_orders = [1, 1, -1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['psnr', 'ssim', 'lpips', 'normals_mae']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names))\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 4986,
     "status": "ok",
     "timestamp": 1656870945508,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "k54UULwA3U-N",
    "outputId": "6543b07e-009a-4f42-c882-bd2ac9771c46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blender_refnerf_453115176 coffee {'psnr': 34.890018711090086, 'ssim': 0.9749827212095261, 'lpips': 0.07588588723912836, 'normals_mae': 11.845668504500003, 'num_hours': 6.575390185535848, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 helmet3 {'psnr': 29.714286947250365, 'ssim': 0.9552837216854095, 'lpips': 0.08357305970042944, 'normals_mae': 36.671281015, 'num_hours': 6.560205667893172, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 musclecar {'psnr': 31.08492564201355, 'ssim': 0.9564637777209282, 'lpips': 0.04047968026250601, 'normals_mae': 15.1815526225, 'num_hours': 6.561444666399807, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 phong_envmap {'psnr': 38.04869948387146, 'ssim': 0.9839657709002495, 'lpips': 0.0813480182737112, 'normals_mae': 4.32499015, 'num_hours': 6.565148490103001, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 teapot2 {'psnr': 45.91226264953613, 'ssim': 0.9964187118411064, 'lpips': 0.009179396553663536, 'normals_mae': 26.659667385, 'num_hours': 6.568492677763366, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 toaster {'psnr': 25.5054944562912, 'ssim': 0.9183712688088417, 'lpips': 0.10447088478133082, 'normals_mae': 43.071832670000006, 'num_hours': 6.571352772368665, 'mega_params': 0.71323}\n",
      "blender_refnerf_453115176 {'psnr': 34.19261464834213, 'ssim': 0.9642476620276769, 'lpips': 0.06582282113512823, 'normals_mae': 22.959165391166664, 'num_hours': 6.567005743343977, 'mega_params': 0.7132300000000001}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/dorverbin/nerf/mipnerf360/'\n",
    "scene_names = ['coffee', 'helmet3', 'musclecar', 'phong_envmap', 'teapot2', 'toaster']\n",
    "metric_names = ['psnr', 'ssim', 'lpips', 'normals_mae']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "models_meta[None] = 'ref-NeRF (from the paper)'\n",
    "models_meta['blender_refnerf_453115176'] = 'ref-NeRF 360'\n",
    "\n",
    "NUM_ITERS = 250000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for model_path in models_meta.keys():\n",
    "\n",
    "  if model_path is None:\n",
    "    # Inject the numbers from the paper.\n",
    "    psnrs = [34.21, 29.68, 30.82, 47.46, 47.90, 25.70]\n",
    "    ssims = [0.974, 0.958, 0.955, 0.995, 0.998, 0.922]\n",
    "    lpips = [0.078, 0.075, 0.041, 0.059, 0.004, 0.095]\n",
    "    nmaes = [12.240, 29.484, 14.927, 1.548, 9.234, 42.870]\n",
    "    train_times = [np.nan]*len(psnrs)\n",
    "    model_sizes = [np.nan]*len(psnrs)\n",
    "    scene_stats = []\n",
    "    for p, s, l, n, tt, ms in zip(psnrs, ssims, lpips, nmaes, train_times, model_sizes):\n",
    "      scene_stats.append({'psnr': p, 'ssim': s, 'lpips': l, 'normals_mae': n, 'num_hours': tt, 'mega_params': ms})\n",
    "    avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "    all_stats.append(scene_stats)\n",
    "    continue;\n",
    "\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS, metric_names=metric_names)\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 74,
     "status": "ok",
     "timestamp": 1656870945704,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "DpWZo3IIRnq4",
    "outputId": "05b1c264-f524-44b9-c447-eb5cc6d9603c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ref-NeRF (from the paper) &    \\cellcolor{red}35.96 &    \\cellcolor{red}0.967 &    \\cellcolor{red}0.059 &    \\cellcolor{red}18.38 &  -  &  -  \\\\\n",
      "ref-NeRF 360              & \\cellcolor{orange}34.19 & \\cellcolor{orange}0.964 & \\cellcolor{orange}0.066 & \\cellcolor{orange}22.96 & 6.57 & 0.7M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 2, 1]\n",
    "rank_order = [1, 1, -1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 56,
     "status": "ok",
     "timestamp": 1656870945900,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "rJpqItPs3hLA",
    "outputId": "8a19973b-2968-4fe3-ed12-3e7d710635b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "psnr\n",
      " & \\textit{coffee} & \\textit{helmet} & \\textit{car} & \\textit{ball} & \\textit{teapot} & \\textit{toaster} \\\\\\hline\n",
      "ref-NeRF (from the paper) & \\cellcolor{orange}34.21 & \\cellcolor{orange}29.68 & \\cellcolor{orange}30.82 &    \\cellcolor{red}47.46 &    \\cellcolor{red}47.90 &    \\cellcolor{red}25.70 \\\\\n",
      "ref-NeRF 360              &    \\cellcolor{red}34.89 &    \\cellcolor{red}29.71 &    \\cellcolor{red}31.08 & \\cellcolor{orange}38.05 & \\cellcolor{orange}45.91 & \\cellcolor{orange}25.51 \\\\\n",
      "\n",
      "ssim\n",
      " & \\textit{coffee} & \\textit{helmet} & \\textit{car} & \\textit{ball} & \\textit{teapot} & \\textit{toaster} \\\\\\hline\n",
      "ref-NeRF (from the paper) & \\cellcolor{orange}0.974 &    \\cellcolor{red}0.958 & \\cellcolor{orange}0.955 &    \\cellcolor{red}0.995 &    \\cellcolor{red}0.998 &    \\cellcolor{red}0.922 \\\\\n",
      "ref-NeRF 360              &    \\cellcolor{red}0.975 & \\cellcolor{orange}0.955 &    \\cellcolor{red}0.956 & \\cellcolor{orange}0.984 & \\cellcolor{orange}0.996 & \\cellcolor{orange}0.918 \\\\\n",
      "\n",
      "lpips\n",
      " & \\textit{coffee} & \\textit{helmet} & \\textit{car} & \\textit{ball} & \\textit{teapot} & \\textit{toaster} \\\\\\hline\n",
      "ref-NeRF (from the paper) & \\cellcolor{orange}0.078 &    \\cellcolor{red}0.075 & \\cellcolor{orange}0.041 &    \\cellcolor{red}0.059 &    \\cellcolor{red}0.004 &    \\cellcolor{red}0.095 \\\\\n",
      "ref-NeRF 360              &    \\cellcolor{red}0.076 & \\cellcolor{orange}0.084 &    \\cellcolor{red}0.040 & \\cellcolor{orange}0.081 & \\cellcolor{orange}0.009 & \\cellcolor{orange}0.104 \\\\\n",
      "\n",
      "normals_mae\n",
      " & \\textit{coffee} & \\textit{helmet} & \\textit{car} & \\textit{ball} & \\textit{teapot} & \\textit{toaster} \\\\\\hline\n",
      "ref-NeRF (from the paper) & \\cellcolor{orange}12.24 &    \\cellcolor{red}29.48 &    \\cellcolor{red}14.93 &    \\cellcolor{red}1.55 &    \\cellcolor{red}9.23 &    \\cellcolor{red}42.87 \\\\\n",
      "ref-NeRF 360              &    \\cellcolor{red}11.85 & \\cellcolor{orange}36.67 & \\cellcolor{orange}15.18 & \\cellcolor{orange}4.32 & \\cellcolor{orange}26.66 & \\cellcolor{orange}43.07 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3, 2]\n",
    "rank_orders = [1, 1, -1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "name_map['coffee'] = 'coffee'\n",
    "name_map['teapot2'] = 'teapot'\n",
    "name_map['musclecar'] = 'car'\n",
    "name_map['phong_envmap'] = 'ball'\n",
    "name_map['helmet3'] = 'helmet'\n",
    "name_map['toaster'] = 'toaster'\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['psnr', 'ssim', 'lpips', 'normals_mae']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names))\n",
    "  print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nXzghNgdJ35g"
   },
   "source": [
    "Reproducing RawNeRF's results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 44245,
     "status": "ok",
     "timestamp": 1656870990258,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "ueL648m2ShTT",
    "outputId": "75855af0-a037-4f22-86b4-304638a38f35"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llff_raw_test_455616468 officetest {'cc_psnr': 23.438732147216797, 'cc_ssim': 0.5584264993667603, 'cc_lpips': 0.4703865349292755, 'num_hours': 5.789938422481767, 'mega_params': 0.61574}\n",
      "llff_raw_test_455616468 pianotest {'cc_psnr': 24.394481658935547, 'cc_ssim': 0.5581468343734741, 'cc_lpips': 0.4914418160915375, 'num_hours': 5.789418753302435, 'mega_params': 0.61574}\n",
      "llff_raw_test_455616468 yuccatest {'cc_psnr': 22.472274780273438, 'cc_ssim': 0.48502984642982483, 'cc_lpips': 0.5376803278923035, 'num_hours': 5.788862314940476, 'mega_params': 0.61574}\n",
      "llff_raw_test_455616468 {'cc_psnr': 23.435162862141926, 'cc_ssim': 0.533867726723353, 'cc_lpips': 0.49983622630437213, 'num_hours': 5.789406496908227, 'mega_params': 0.61574}\n"
     ]
    }
   ],
   "source": [
    "root_folder = '/cns/lu-d/home/buff/bmild/nerf/mipnerf360'\n",
    "scene_names = ['officetest', 'pianotest', 'yuccatest']\n",
    "\n",
    "models_meta = {} # folder : latex_name\n",
    "models_meta[None] = 'rawNeRF (from the paper)'\n",
    "models_meta['llff_raw_test_455616468'] = 'rawNeRF'\n",
    "\n",
    "NUM_ITERS = 500000\n",
    "\n",
    "all_stats = []\n",
    "avg_stats = []\n",
    "for model_path in models_meta.keys():\n",
    "\n",
    "  if model_path is None:\n",
    "    # Inject the numbers from the paper.\n",
    "    psnrs = [23.42711075794648, 24.387560024128817, 22.3921655141463]\n",
    "    ssims = [0.5596604, 0.55790526, 0.48582202]\n",
    "    lpips = [0.47882235, 0.49443525, 0.5339949]\n",
    "    train_times = [np.nan]*len(psnrs)\n",
    "    model_sizes = [np.nan]*len(psnrs)\n",
    "    scene_stats = []\n",
    "    for p, s, l, tt, ms in zip(psnrs, ssims, lpips, train_times, model_sizes):\n",
    "      scene_stats.append({'cc_psnr': p, 'cc_ssim': s, 'cc_lpips': l, 'num_hours': tt, 'mega_params': ms})\n",
    "    avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "    all_stats.append(scene_stats)\n",
    "    continue;\n",
    "\n",
    "  scene_stats = []\n",
    "  for scene_name in scene_names:\n",
    "    folder = os.path.join(root_folder, model_path, scene_name)\n",
    "    stats = scrape_folder(folder, NUM_ITERS, metric_names = ['cc_psnr', 'cc_ssim', 'cc_lpips'])\n",
    "    print(model_path, scene_name, stats)\n",
    "    scene_stats.append(stats)\n",
    "  avg_stats.append({k: type(scene_stats[0][k])(np.mean([s[k] for s in scene_stats])) for k in scene_stats[0].keys()})\n",
    "  all_stats.append(scene_stats)\n",
    "  print(model_path, avg_stats[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 54,
     "status": "ok",
     "timestamp": 1656870990431,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "0Qgef6LHJ841",
    "outputId": "b9fcb848-1134-4cd9-b239-933dec1ac454"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rawNeRF (from the paper) & \\cellcolor{orange}23.40 &    \\cellcolor{red}0.534 & \\cellcolor{orange}0.502 &  -  &  -  \\\\\n",
      "rawNeRF                  &    \\cellcolor{red}23.44 &    \\cellcolor{red}0.534 &    \\cellcolor{red}0.500 & 5.79 & 0.6M \\\\\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "data = np.stack([list(s.values()) for s in avg_stats])\n",
    "precisions = [2, 3, 3, 2, 1]\n",
    "rank_order = [1, 1, -1, 0, 0]  # +1 = higher is better, -1 = lower is better, 0 = do not color code\n",
    "suffixes = ['', '', '', '', 'M']\n",
    "render_table(names, data, precisions, rank_order, suffixes=suffixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 70,
     "status": "ok",
     "timestamp": 1656870990613,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": -60
    },
    "id": "c594nENnJ-bB",
    "outputId": "46f0885d-b225-4484-849d-ec20818a7e88"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cc_psnr\n",
      " & \\textit{officetest} & \\textit{pianotest} & \\textit{yuccatest} \\\\\\hline\n",
      "\\hline\n",
      "rawNeRF (from the paper) & \\cellcolor{orange}23.43 &    \\cellcolor{red}24.39 & \\cellcolor{orange}22.39 \\\\\n",
      "rawNeRF                  &    \\cellcolor{red}23.44 &    \\cellcolor{red}24.39 &    \\cellcolor{red}22.47 \\\\\n",
      "\n",
      "cc_ssim\n",
      " & \\textit{officetest} & \\textit{pianotest} & \\textit{yuccatest} \\\\\\hline\n",
      "\\hline\n",
      "rawNeRF (from the paper) &    \\cellcolor{red}0.560 &    \\cellcolor{red}0.558 &    \\cellcolor{red}0.486 \\\\\n",
      "rawNeRF                  & \\cellcolor{orange}0.558 &    \\cellcolor{red}0.558 & \\cellcolor{orange}0.485 \\\\\n",
      "\n",
      "cc_lpips\n",
      " & \\textit{officetest} & \\textit{pianotest} & \\textit{yuccatest} \\\\\\hline\n",
      "\\hline\n",
      "rawNeRF (from the paper) & \\cellcolor{orange}0.479 & \\cellcolor{orange}0.494 &    \\cellcolor{red}0.534 \\\\\n",
      "rawNeRF                  &    \\cellcolor{red}0.470 &    \\cellcolor{red}0.491 & \\cellcolor{orange}0.538 \\\\\n",
      "\n"
     ]
    }
   ],
   "source": [
    "names = list(models_meta.values())\n",
    "\n",
    "precisions = [2, 3, 3]\n",
    "rank_orders = [1, 1, -1]\n",
    "\n",
    "name_map = {s: s for s in scene_names}\n",
    "pretty_scene_names = [name_map[s] for s in scene_names]\n",
    "\n",
    "for i_metric, metric in enumerate(['cc_psnr', 'cc_ssim', 'cc_lpips']):\n",
    "  print(metric)\n",
    "  precision = precisions[i_metric]\n",
    "  rank_order = rank_orders[i_metric]\n",
    "\n",
    "  print(' & ' + ' & '.join(['\\\\textit{' + s + '}' for s in pretty_scene_names]) + ' \\\\\\\\\\\\hline')\n",
    "  data = np.array([np.array([s[metric] for s in scene_stats]) for scene_stats in all_stats])\n",
    "  render_table(names, data, [precision] * len(scene_names), [rank_order] * len(scene_names), hlines = [len(names)-2])\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "l2kJLXgyMWWM"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "last_runtime": {
    "build_target": "//learning/deepmind/public/tools/ml_python:ml_notebook",
    "kind": "private"
   },
   "name": "generate_tables.ipynb",
   "provenance": [
    {
     "file_id": "/piper/depot/google3/googlex/gcam/buff/mipnerf360/scripts/generate_tables.ipynb?workspaceId=dorverbin:refnerf_hybrid::citc",
     "timestamp": 1654708265431
    },
    {
     "file_id": "/piper/depot/google3/experimental/users/barron/prob_nerf/scripts/eval_single.ipynb?workspaceId=barron:mipnerf360_paper::citc",
     "timestamp": 1635265902094
    },
    {
     "file_id": "/piper/depot/google3/experimental/users/barron/prob_nerf/scripts/Pre_NeRF_Eval_multi.ipynb?workspaceId=barron:jaxnerf_mono5::citc",
     "timestamp": 1614394543651
    },
    {
     "file_id": "/piper/depot/google3/experimental/users/barron/prob_nerf/scripts/Pre_NeRF_Eval.ipynb?workspaceId=barron:jaxnerf_mono5::citc",
     "timestamp": 1614038274387
    },
    {
     "file_id": "10opVizeODokMJ10R7hwq7qVyLmYZx_ZA",
     "timestamp": 1613166364224
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
